{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26. Jefferson Monticello Thomas age at began building of the \n",
      "\n",
      "[[0 0 0 1 0 0 0 0 0 0]\n",
      " [0 1 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 1 0 0 0]\n",
      " [0 0 0 0 0 0 0 1 0 0]\n",
      " [0 0 1 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 1 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0 1]\n",
      " [0 0 0 0 1 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 1 0]\n",
      " [1 0 0 0 0 0 0 0 0 0]] \n",
      "\n",
      "       Thomas  Jefferson  began  building  Monticello  at  the\\  age  of  26.\n",
      "sent0       1          1      1         1           1   1     1    1   1    1\n",
      "sent1       0          0      0         0           0   0     0    0   0    0\n",
      "sent2       0          0      0         0           0   0     0    0   0    0\n",
      "sent3       0          0      0         0           1   0     0    0   0    0 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "sentence = \"\"\"Thomas Jefferson began building Monticello at the age of 26.\"\"\"\n",
    "token_sequence = str.split(sentence) \n",
    "vocab = sorted(set(token_sequence)) \n",
    "# print(', '.join(vocab) )\n",
    "num_tokens = len(token_sequence)\n",
    "vocab_size = len(vocab)\n",
    "onehot_vectors = np.zeros((num_tokens, vocab_size), int) \n",
    "for i, word in enumerate(token_sequence): onehot_vectors[i, vocab.index(word)] = 1 \n",
    "print(' '.join(vocab),'\\n')\n",
    "print(onehot_vectors,'\\n')\n",
    "pd.DataFrame(onehot_vectors, columns=vocab)\n",
    "df = pd.DataFrame(onehot_vectors, columns=vocab)\n",
    "df[df == 0] = ''\n",
    "# print(df)\n",
    "num_rows = 3000 * 3500 * 15\n",
    "num_bytes = num_rows * 1000000\n",
    "sentence_bow = {}\n",
    "for token in sentence.split(): \n",
    "    sentence_bow[token] = 1\n",
    "sorted(sentence_bow.items())\n",
    "df = pd.DataFrame(pd.Series(dict([(token, 1) for token in sentence.split()])), columns=['sent']).T\n",
    "# print(df)\n",
    "sentences = \"\"\"Thomas Jefferson began building Monticello at the\\ age of 26.\\n\"\"\"\n",
    "sentences += \"\"\"Construction was done mostly by local masons and\\ carpenters.\\n\"\"\"\n",
    "sentences += \"He moved into the South Pavilion in 1770.\\n\"\n",
    "sentences += \"\"\"Turning Monticello into a neoclassical masterpiece\\ was Jefferson's obsession.\"\"\"\n",
    "corpus = {}\n",
    "for i, sent in enumerate(sentences.split('\\n')):\n",
    "    corpus['sent{}'.format(i)] = dict((tok, 1) for tok in sent.split())\n",
    "df = pd.DataFrame.from_records(corpus).fillna(0).astype(int).T\n",
    "print(df[df.columns[:10]],'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "#度量词袋之间的重合度#\n",
      "[('Monticello', 1)]\n",
      "\n",
      "#标点符号的处理#\n",
      "    re库 \n",
      " ['Thomas', 'Jefferson', 'began', 'building', 'Monticello', 'at', 'the', 'age', 'of', '26']\n",
      "    nltk库 \n",
      " ['Monticello', 'was', \"n't\", 'designated', 'as', 'UNESCO', 'World', 'Heritage', 'Site', 'until', '1987', '.']\n",
      "\n",
      "#缩略语#\n",
      "['RT', 'Best', 'day', 'everrr', 'at', 'Monticello', '.', 'Awesommmeee', 'day', ':*)']\n"
     ]
    }
   ],
   "source": [
    "df = df.T\n",
    "print(\"\\n#度量词袋之间的重合度#\")\n",
    "df.sent0.dot(df.sent1)\n",
    "df.sent0.dot(df.sent2)\n",
    "df.sent0.dot(df.sent3)\n",
    "print([(k, v) for (k, v) in (df.sent0 & df.sent3).items() if v])\n",
    "print(\"\\n#标点符号的处理#\")\n",
    "import re\n",
    "sentence = \"\"\"Thomas Jefferson began building Monticello at the age of 26.\"\"\"\n",
    "tokens = re.split(r'[-\\s.,;!?]+', sentence)\n",
    "#print(tokens)\n",
    "pattern = re.compile(r\"([-\\s.,;!?])+\")\n",
    "sentence = \"\"\"Thomas Jefferson began building Monticello at the age of 26.\"\"\"\n",
    "tokens = pattern.split(sentence)\n",
    "print(\"    re库 \\n\", [x for x in tokens if x and x not in '- \\t\\n.,;!?'])\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "tokenizer = RegexpTokenizer(r'\\w+|$[0-9.]+|\\S+')\n",
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "sentence = \"\"\"Monticello wasn't designated as UNESCO World Heritage Site until 1987.\"\"\"\n",
    "tokenizer = TreebankWordTokenizer()\n",
    "print(\"    nltk库 \\n\",tokenizer.tokenize(sentence))\n",
    "print(\"\\n#缩略语#\")\n",
    "from nltk.tokenize.casual import casual_tokenize\n",
    "message = \"\"\"RT @TJMonticello Best day everrrrrrr at Monticello. Awesommmmmmeeeeeeee day :*)\"\"\"\n",
    "# casual_tokenize(message)\n",
    "print(casual_tokenize(message, reduce_len=True, strip_handles=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "#ngram词词汇表#\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Error loading stopwords: <urlopen error [Errno 11004]\n",
      "[nltk_data]     getaddrinfo failed>\n"
     ]
    },
    {
     "ename": "LookupError",
     "evalue": "\n**********************************************************************\n  Resource \u001b[93mstopwords\u001b[0m not found.\n  Please use the NLTK Downloader to obtain the resource:\n\n  \u001b[31m>>> import nltk\n  >>> nltk.download('stopwords')\n  \u001b[0m\n  For more information see: https://www.nltk.org/data.html\n\n  Attempted to load \u001b[93mcorpora/stopwords\u001b[0m\n\n  Searched in:\n    - 'C:\\\\Users\\\\SIS/nltk_data'\n    - 'C:\\\\Users\\\\SIS\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python38\\\\nltk_data'\n    - 'C:\\\\Users\\\\SIS\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python38\\\\share\\\\nltk_data'\n    - 'C:\\\\Users\\\\SIS\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python38\\\\lib\\\\nltk_data'\n    - 'C:\\\\Users\\\\SIS\\\\AppData\\\\Roaming\\\\nltk_data'\n    - 'C:\\\\nltk_data'\n    - 'D:\\\\nltk_data'\n    - 'E:\\\\nltk_data'\n**********************************************************************\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mLookupError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\nltk\\corpus\\util.py\u001b[0m in \u001b[0;36m__load\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     83\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 84\u001b[1;33m                     \u001b[0mroot\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mf\"{self.subdir}/{zip_name}\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     85\u001b[0m                 \u001b[1;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\nltk\\data.py\u001b[0m in \u001b[0;36mfind\u001b[1;34m(resource_name, paths)\u001b[0m\n\u001b[0;32m    582\u001b[0m     \u001b[0mresource_not_found\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34mf\"\\n{sep}\\n{msg}\\n{sep}\\n\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 583\u001b[1;33m     \u001b[1;32mraise\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresource_not_found\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    584\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mLookupError\u001b[0m: \n**********************************************************************\n  Resource \u001b[93mstopwords\u001b[0m not found.\n  Please use the NLTK Downloader to obtain the resource:\n\n  \u001b[31m>>> import nltk\n  >>> nltk.download('stopwords')\n  \u001b[0m\n  For more information see: https://www.nltk.org/data.html\n\n  Attempted to load \u001b[93mcorpora/stopwords.zip/stopwords/\u001b[0m\n\n  Searched in:\n    - 'C:\\\\Users\\\\SIS/nltk_data'\n    - 'C:\\\\Users\\\\SIS\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python38\\\\nltk_data'\n    - 'C:\\\\Users\\\\SIS\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python38\\\\share\\\\nltk_data'\n    - 'C:\\\\Users\\\\SIS\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python38\\\\lib\\\\nltk_data'\n    - 'C:\\\\Users\\\\SIS\\\\AppData\\\\Roaming\\\\nltk_data'\n    - 'C:\\\\nltk_data'\n    - 'D:\\\\nltk_data'\n    - 'E:\\\\nltk_data'\n**********************************************************************\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mLookupError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-bb74298fb7dc>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnltk\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mnltk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'stopwords'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mstop_words\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstopwords\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwords\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'english'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstop_words\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstop_words\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m7\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\nltk\\corpus\\util.py\u001b[0m in \u001b[0;36m__getattr__\u001b[1;34m(self, attr)\u001b[0m\n\u001b[0;32m    119\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"LazyCorpusLoader object has no attribute '__bases__'\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    120\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 121\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__load\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    122\u001b[0m         \u001b[1;31m# This looks circular, but its not, since __load() changes our\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    123\u001b[0m         \u001b[1;31m# __class__ to something new:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\nltk\\corpus\\util.py\u001b[0m in \u001b[0;36m__load\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     84\u001b[0m                     \u001b[0mroot\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mf\"{self.subdir}/{zip_name}\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     85\u001b[0m                 \u001b[1;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 86\u001b[1;33m                     \u001b[1;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     87\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     88\u001b[0m         \u001b[1;31m# Load the corpus.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\nltk\\corpus\\util.py\u001b[0m in \u001b[0;36m__load\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     79\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     80\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 81\u001b[1;33m                 \u001b[0mroot\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mf\"{self.subdir}/{self.__name}\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     82\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     83\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python38\\lib\\site-packages\\nltk\\data.py\u001b[0m in \u001b[0;36mfind\u001b[1;34m(resource_name, paths)\u001b[0m\n\u001b[0;32m    581\u001b[0m     \u001b[0msep\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"*\"\u001b[0m \u001b[1;33m*\u001b[0m \u001b[1;36m70\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    582\u001b[0m     \u001b[0mresource_not_found\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34mf\"\\n{sep}\\n{msg}\\n{sep}\\n\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 583\u001b[1;33m     \u001b[1;32mraise\u001b[0m \u001b[0mLookupError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresource_not_found\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    584\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    585\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mLookupError\u001b[0m: \n**********************************************************************\n  Resource \u001b[93mstopwords\u001b[0m not found.\n  Please use the NLTK Downloader to obtain the resource:\n\n  \u001b[31m>>> import nltk\n  >>> nltk.download('stopwords')\n  \u001b[0m\n  For more information see: https://www.nltk.org/data.html\n\n  Attempted to load \u001b[93mcorpora/stopwords\u001b[0m\n\n  Searched in:\n    - 'C:\\\\Users\\\\SIS/nltk_data'\n    - 'C:\\\\Users\\\\SIS\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python38\\\\nltk_data'\n    - 'C:\\\\Users\\\\SIS\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python38\\\\share\\\\nltk_data'\n    - 'C:\\\\Users\\\\SIS\\\\AppData\\\\Local\\\\Programs\\\\Python\\\\Python38\\\\lib\\\\nltk_data'\n    - 'C:\\\\Users\\\\SIS\\\\AppData\\\\Roaming\\\\nltk_data'\n    - 'C:\\\\nltk_data'\n    - 'D:\\\\nltk_data'\n    - 'E:\\\\nltk_data'\n**********************************************************************\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n#ngram词词汇表#\")\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "stop_words = nltk.corpus.stopwords.words('english')\n",
    "print(len(stop_words))\n",
    "print(stop_words[:7])\n",
    "print(\"\\n#词汇表规范化#\")\n",
    "tokens = ['House', 'Visitor', 'Center']\n",
    "normalized_tokens = [x.lower() for x in tokens]\n",
    "print(normalized_tokens)\n",
    "def stem(phrase):\n",
    "    return ' '.join([re.findall('^(.*ss|.*?)(s)?$', word)[0][0].strip(\"'\") for word in phrase.lower().split()])\n",
    "print(stem(\"Doctor House's calls\"))\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "stemmer = PorterStemmer()\n",
    "print(' '.join([stemmer.stem(w).strip(\"'\") for w in \"dish washer's washed dishes\".split()]))\n",
    "nltk.download('wordnet')\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "print(lemmatizer.lemmatize(\"better\"),\n",
    "      lemmatizer.lemmatize(\"better\", pos=\"a\"),\n",
    "      lemmatizer.lemmatize(\"good\", pos=\"a\"),\n",
    "      lemmatizer.lemmatize(\"goods\", pos=\"a\"),\n",
    "      lemmatizer.lemmatize(\"goods\", pos=\"n\"),\n",
    "      lemmatizer.lemmatize(\"goodness\", pos=\"n\"),\n",
    "      lemmatizer.lemmatize(\"best\", pos=\"a\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "#文本情感分析#\n",
      "{'neg': 0.0, 'neu': 0.661, 'pos': 0.339, 'compound': 0.6249}\n",
      "{'neg': 0.0, 'neu': 0.737, 'pos': 0.263, 'compound': 0.431}\n",
      "+0.9428: Absolutely perfect! Love it! :-) :-) :-)\n",
      "-0.8768: Horrible! Completely useless. :(\n",
      "-0.1531: It was OK. Some good and some bad things.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>The</th>\n",
       "      <th>Rock</th>\n",
       "      <th>is</th>\n",
       "      <th>destined</th>\n",
       "      <th>to</th>\n",
       "      <th>be</th>\n",
       "      <th>the</th>\n",
       "      <th>21st</th>\n",
       "      <th>Century's</th>\n",
       "      <th>new</th>\n",
       "      <th>...</th>\n",
       "      <th>Schwarzenegger</th>\n",
       "      <th>,</th>\n",
       "      <th>Jean</th>\n",
       "      <th>Claud</th>\n",
       "      <th>Van</th>\n",
       "      <th>Damme</th>\n",
       "      <th>or</th>\n",
       "      <th>Steven</th>\n",
       "      <th>Segal</th>\n",
       "      <th>.</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   The  Rock  is  destined  to  be  the  21st  Century's  new  ...  \\\n",
       "0    1     1   1         1   2   1    1     1          1    1  ...   \n",
       "1    2     0   1         0   0   0    1     0          0    0  ...   \n",
       "2    0     0   0         0   0   0    0     0          0    0  ...   \n",
       "3    0     0   1         0   4   0    1     0          0    0  ...   \n",
       "4    0     0   0         0   0   0    0     0          0    0  ...   \n",
       "\n",
       "   Schwarzenegger  ,  Jean  Claud  Van  Damme  or  Steven  Segal  .  \n",
       "0               1  1     1      1    1      1   1       1      1  1  \n",
       "1               0  0     0      0    0      0   0       0      0  4  \n",
       "2               0  0     0      0    0      0   0       0      0  0  \n",
       "3               0  1     0      0    0      0   0       0      0  1  \n",
       "4               0  1     0      0    0      0   0       0      0  1  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"\\n\\n#文本情感分析#\")\n",
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
    "sa = SentimentIntensityAnalyzer()\n",
    "print(sa.polarity_scores(text= \"Python is very readable and it's great for NLP.\"))\n",
    "print( sa.polarity_scores(text=\"Python is not a bad choice for most applications.\"))\n",
    "corpus = [\"Absolutely perfect! Love it! :-) :-) :-)\",\n",
    "          \"Horrible! Completely useless. :(\",\n",
    "          \"It was OK. Some good and some bad things.\"]\n",
    "for doc in corpus:\n",
    "    scores = sa.polarity_scores(doc)\n",
    "    print('{:+}: {}'.format(scores['compound'], doc))\n",
    "from nlpia.data.loaders import get_data\n",
    "movies = get_data('hutto_movies')\n",
    "movies.head().round(2)\n",
    "movies.describe().round(2)\n",
    "import pandas as pd\n",
    "pd.set_option('display.width', 75)\n",
    "from nltk.tokenize import casual_tokenize\n",
    "bags_of_words = []\n",
    "from collections import Counter\n",
    "for text in movies.text:\n",
    "    bags_of_words.append(Counter(casual_tokenize(text)))\n",
    "df_bows = pd.DataFrame.from_records(bags_of_words)\n",
    "df_bows = df_bows.fillna(0).astype(int)\n",
    "#df_bows.shape\n",
    "#df_bows.head()\n",
    "df_bows.head()[list(bags_of_words[0].keys())]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>predicted_sentiment</th>\n",
       "      <th>sentiment_ispositive</th>\n",
       "      <th>error</th>\n",
       "      <th>predicted_ispositive</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.266667</td>\n",
       "      <td>The Rock is destined to be the 21st Century's ...</td>\n",
       "      <td>-2.511515</td>\n",
       "      <td>1</td>\n",
       "      <td>4.778181</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.533333</td>\n",
       "      <td>The gorgeously elaborate continuation of ''The...</td>\n",
       "      <td>-3.999904</td>\n",
       "      <td>1</td>\n",
       "      <td>7.533238</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>-0.600000</td>\n",
       "      <td>Effective but too tepid biopic</td>\n",
       "      <td>3.655976</td>\n",
       "      <td>0</td>\n",
       "      <td>4.255976</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.466667</td>\n",
       "      <td>If you sometimes like to go to the movies to h...</td>\n",
       "      <td>-1.940954</td>\n",
       "      <td>1</td>\n",
       "      <td>3.407621</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.733333</td>\n",
       "      <td>Emerges as something rare, an issue movie that...</td>\n",
       "      <td>-3.910373</td>\n",
       "      <td>1</td>\n",
       "      <td>5.643706</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10601</td>\n",
       "      <td>-0.062500</td>\n",
       "      <td>Well made but mush hearted.</td>\n",
       "      <td>3.166489</td>\n",
       "      <td>0</td>\n",
       "      <td>3.228989</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10602</td>\n",
       "      <td>-1.500000</td>\n",
       "      <td>A real snooze.</td>\n",
       "      <td>1.056805</td>\n",
       "      <td>0</td>\n",
       "      <td>2.556805</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10603</td>\n",
       "      <td>-0.625000</td>\n",
       "      <td>No surprises.</td>\n",
       "      <td>1.481449</td>\n",
       "      <td>0</td>\n",
       "      <td>2.106449</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10604</td>\n",
       "      <td>1.437500</td>\n",
       "      <td>We’ve seen the hippie turned yuppie plot befor...</td>\n",
       "      <td>-3.988988</td>\n",
       "      <td>1</td>\n",
       "      <td>5.426488</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10605</td>\n",
       "      <td>-1.812500</td>\n",
       "      <td>Her fans walked out muttering words like ''hor...</td>\n",
       "      <td>3.997954</td>\n",
       "      <td>0</td>\n",
       "      <td>5.810454</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10605 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       sentiment                                               text  \\\n",
       "id                                                                    \n",
       "1       2.266667  The Rock is destined to be the 21st Century's ...   \n",
       "2       3.533333  The gorgeously elaborate continuation of ''The...   \n",
       "3      -0.600000                     Effective but too tepid biopic   \n",
       "4       1.466667  If you sometimes like to go to the movies to h...   \n",
       "5       1.733333  Emerges as something rare, an issue movie that...   \n",
       "...          ...                                                ...   \n",
       "10601  -0.062500                        Well made but mush hearted.   \n",
       "10602  -1.500000                                     A real snooze.   \n",
       "10603  -0.625000                                      No surprises.   \n",
       "10604   1.437500  We’ve seen the hippie turned yuppie plot befor...   \n",
       "10605  -1.812500  Her fans walked out muttering words like ''hor...   \n",
       "\n",
       "       predicted_sentiment  sentiment_ispositive     error  \\\n",
       "id                                                           \n",
       "1                -2.511515                     1  4.778181   \n",
       "2                -3.999904                     1  7.533238   \n",
       "3                 3.655976                     0  4.255976   \n",
       "4                -1.940954                     1  3.407621   \n",
       "5                -3.910373                     1  5.643706   \n",
       "...                    ...                   ...       ...   \n",
       "10601             3.166489                     0  3.228989   \n",
       "10602             1.056805                     0  2.556805   \n",
       "10603             1.481449                     0  2.106449   \n",
       "10604            -3.988988                     1  5.426488   \n",
       "10605             3.997954                     0  5.810454   \n",
       "\n",
       "       predicted_ispositive  \n",
       "id                           \n",
       "1                         0  \n",
       "2                         0  \n",
       "3                         1  \n",
       "4                         0  \n",
       "5                         0  \n",
       "...                     ...  \n",
       "10601                     1  \n",
       "10602                     1  \n",
       "10603                     1  \n",
       "10604                     0  \n",
       "10605                     1  \n",
       "\n",
       "[10605 rows x 6 columns]"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "nb = MultinomialNB()\n",
    "nb = nb.fit(df_bows, movies.sentiment > 0)\n",
    "nb\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>predicted_sentiment</th>\n",
       "      <th>sentiment_ispositive</th>\n",
       "      <th>error</th>\n",
       "      <th>predicted_ispositive</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.266667</td>\n",
       "      <td>The Rock is destined to be the 21st Century's ...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4.778181</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.533333</td>\n",
       "      <td>The gorgeously elaborate continuation of ''The...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.533238</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>-0.600000</td>\n",
       "      <td>Effective but too tepid biopic</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4.255976</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.466667</td>\n",
       "      <td>If you sometimes like to go to the movies to h...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.407621</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.733333</td>\n",
       "      <td>Emerges as something rare, an issue movie that...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5.643706</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10601</td>\n",
       "      <td>-0.062500</td>\n",
       "      <td>Well made but mush hearted.</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.228989</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10602</td>\n",
       "      <td>-1.500000</td>\n",
       "      <td>A real snooze.</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.556805</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10603</td>\n",
       "      <td>-0.625000</td>\n",
       "      <td>No surprises.</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.106449</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10604</td>\n",
       "      <td>1.437500</td>\n",
       "      <td>We’ve seen the hippie turned yuppie plot befor...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5.426488</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10605</td>\n",
       "      <td>-1.812500</td>\n",
       "      <td>Her fans walked out muttering words like ''hor...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5.810454</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10605 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       sentiment                                               text  \\\n",
       "id                                                                    \n",
       "1       2.266667  The Rock is destined to be the 21st Century's ...   \n",
       "2       3.533333  The gorgeously elaborate continuation of ''The...   \n",
       "3      -0.600000                     Effective but too tepid biopic   \n",
       "4       1.466667  If you sometimes like to go to the movies to h...   \n",
       "5       1.733333  Emerges as something rare, an issue movie that...   \n",
       "...          ...                                                ...   \n",
       "10601  -0.062500                        Well made but mush hearted.   \n",
       "10602  -1.500000                                     A real snooze.   \n",
       "10603  -0.625000                                      No surprises.   \n",
       "10604   1.437500  We’ve seen the hippie turned yuppie plot befor...   \n",
       "10605  -1.812500  Her fans walked out muttering words like ''hor...   \n",
       "\n",
       "       predicted_sentiment  sentiment_ispositive     error  \\\n",
       "id                                                           \n",
       "1                        1                     1  4.778181   \n",
       "2                        1                     1  7.533238   \n",
       "3                        1                     1  4.255976   \n",
       "4                        1                     1  3.407621   \n",
       "5                        1                     1  5.643706   \n",
       "...                    ...                   ...       ...   \n",
       "10601                    1                     1  3.228989   \n",
       "10602                    1                     1  2.556805   \n",
       "10603                    1                     1  2.106449   \n",
       "10604                    1                     1  5.426488   \n",
       "10605                    1                     1  5.810454   \n",
       "\n",
       "       predicted_ispositive  \n",
       "id                           \n",
       "1                         0  \n",
       "2                         0  \n",
       "3                         1  \n",
       "4                         0  \n",
       "5                         0  \n",
       "...                     ...  \n",
       "10601                     1  \n",
       "10602                     1  \n",
       "10603                     1  \n",
       "10604                     0  \n",
       "10605                     1  \n",
       "\n",
       "[10605 rows x 6 columns]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies['predicted_sentiment'] = 1\n",
    "movies['sentiment_ispositive'] = 1\n",
    "movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>predicted_sentiment</th>\n",
       "      <th>sentiment_ispositive</th>\n",
       "      <th>error</th>\n",
       "      <th>predicted_ispositive</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.266667</td>\n",
       "      <td>The Rock is destined to be the 21st Century's ...</td>\n",
       "      <td>-2.511515</td>\n",
       "      <td>1</td>\n",
       "      <td>4.778181</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.533333</td>\n",
       "      <td>The gorgeously elaborate continuation of ''The...</td>\n",
       "      <td>-3.999904</td>\n",
       "      <td>1</td>\n",
       "      <td>7.533238</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>-0.600000</td>\n",
       "      <td>Effective but too tepid biopic</td>\n",
       "      <td>3.655976</td>\n",
       "      <td>0</td>\n",
       "      <td>4.255976</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.466667</td>\n",
       "      <td>If you sometimes like to go to the movies to h...</td>\n",
       "      <td>-1.940954</td>\n",
       "      <td>1</td>\n",
       "      <td>3.407621</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>1.733333</td>\n",
       "      <td>Emerges as something rare, an issue movie that...</td>\n",
       "      <td>-3.910373</td>\n",
       "      <td>1</td>\n",
       "      <td>5.643706</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10601</td>\n",
       "      <td>-0.062500</td>\n",
       "      <td>Well made but mush hearted.</td>\n",
       "      <td>3.166489</td>\n",
       "      <td>0</td>\n",
       "      <td>3.228989</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10602</td>\n",
       "      <td>-1.500000</td>\n",
       "      <td>A real snooze.</td>\n",
       "      <td>1.056805</td>\n",
       "      <td>0</td>\n",
       "      <td>2.556805</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10603</td>\n",
       "      <td>-0.625000</td>\n",
       "      <td>No surprises.</td>\n",
       "      <td>1.481449</td>\n",
       "      <td>0</td>\n",
       "      <td>2.106449</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10604</td>\n",
       "      <td>1.437500</td>\n",
       "      <td>We’ve seen the hippie turned yuppie plot befor...</td>\n",
       "      <td>-3.988988</td>\n",
       "      <td>1</td>\n",
       "      <td>5.426488</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10605</td>\n",
       "      <td>-1.812500</td>\n",
       "      <td>Her fans walked out muttering words like ''hor...</td>\n",
       "      <td>3.997954</td>\n",
       "      <td>0</td>\n",
       "      <td>5.810454</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10605 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       sentiment                                               text  \\\n",
       "id                                                                    \n",
       "1       2.266667  The Rock is destined to be the 21st Century's ...   \n",
       "2       3.533333  The gorgeously elaborate continuation of ''The...   \n",
       "3      -0.600000                     Effective but too tepid biopic   \n",
       "4       1.466667  If you sometimes like to go to the movies to h...   \n",
       "5       1.733333  Emerges as something rare, an issue movie that...   \n",
       "...          ...                                                ...   \n",
       "10601  -0.062500                        Well made but mush hearted.   \n",
       "10602  -1.500000                                     A real snooze.   \n",
       "10603  -0.625000                                      No surprises.   \n",
       "10604   1.437500  We’ve seen the hippie turned yuppie plot befor...   \n",
       "10605  -1.812500  Her fans walked out muttering words like ''hor...   \n",
       "\n",
       "       predicted_sentiment  sentiment_ispositive     error  \\\n",
       "id                                                           \n",
       "1                -2.511515                     1  4.778181   \n",
       "2                -3.999904                     1  7.533238   \n",
       "3                 3.655976                     0  4.255976   \n",
       "4                -1.940954                     1  3.407621   \n",
       "5                -3.910373                     1  5.643706   \n",
       "...                    ...                   ...       ...   \n",
       "10601             3.166489                     0  3.228989   \n",
       "10602             1.056805                     0  2.556805   \n",
       "10603             1.481449                     0  2.106449   \n",
       "10604            -3.988988                     1  5.426488   \n",
       "10605             3.997954                     0  5.810454   \n",
       "\n",
       "       predicted_ispositive  \n",
       "id                           \n",
       "1                         0  \n",
       "2                         0  \n",
       "3                         1  \n",
       "4                         0  \n",
       "5                         0  \n",
       "...                     ...  \n",
       "10601                     1  \n",
       "10602                     1  \n",
       "10603                     1  \n",
       "10604                     0  \n",
       "10605                     1  \n",
       "\n",
       "[10605 rows x 6 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies['predicted_sentiment'] = nb.predict_proba(df_bows) * 8 - 4\n",
    "movies['error'] = (movies.predicted_sentiment - movies.sentiment).abs()\n",
    "round(movies['error'].mean(),1)\n",
    "movies['sentiment_ispositive'] = (movies.sentiment > 0).astype(int)\n",
    "movies['predicted_ispositive'] = (movies.predicted_sentiment > 0).astype(int)\n",
    "movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/luocong/opt/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py:7123: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  sort=sort,\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(10605, 20756)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies['''sentiment predicted_sentiment sentiment_ispositive predicted_ispositive'''.split()].head(8)\n",
    "products = get_data('hutto_products')\n",
    "bags_of_words = []\n",
    "for text in products.text:\n",
    "    bags_of_words.append(Counter(casual_tokenize(text)))\n",
    "df_product_bows = pd.DataFrame.from_records(bags_of_words)\n",
    "df_product_bows = df_product_bows.fillna(0).astype(int)\n",
    "df_all_bows = df_bows.append(df_product_bows)\n",
    "df_all_bows.columns\n",
    "df_product_bows = df_all_bows.iloc[len(movies):][df_bows.columns]\n",
    "df_product_bows.shape\n",
    "df_bows.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>The</th>\n",
       "      <th>Rock</th>\n",
       "      <th>is</th>\n",
       "      <th>destined</th>\n",
       "      <th>to</th>\n",
       "      <th>be</th>\n",
       "      <th>the</th>\n",
       "      <th>21st</th>\n",
       "      <th>Century's</th>\n",
       "      <th>new</th>\n",
       "      <th>...</th>\n",
       "      <th>Ill</th>\n",
       "      <th>slummer</th>\n",
       "      <th>Rashomon</th>\n",
       "      <th>dipsticks</th>\n",
       "      <th>Bearable</th>\n",
       "      <th>Staggeringly</th>\n",
       "      <th>’</th>\n",
       "      <th>ve</th>\n",
       "      <th>muttering</th>\n",
       "      <th>dissing</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3541</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3542</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3543</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3544</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3545</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3546 rows × 20756 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      The  Rock  is  destined  to  be  the  21st  Century's  new  ...  \\\n",
       "0     1.0   1.0   0       1.0   0   0    0     0        1.0    0  ...   \n",
       "1     1.0   1.0   0       1.0   0   0    0     0        1.0    0  ...   \n",
       "2     1.0   1.0   0       1.0   0   0    0     0        1.0    0  ...   \n",
       "3     1.0   1.0   0       1.0   0   0    0     0        1.0    0  ...   \n",
       "4     1.0   1.0   0       1.0   1   0    2     0        1.0    0  ...   \n",
       "...   ...   ...  ..       ...  ..  ..  ...   ...        ...  ...  ...   \n",
       "3541  1.0   1.0   0       1.0   1   0    1     0        1.0    0  ...   \n",
       "3542  1.0   1.0   0       1.0   0   0    0     0        1.0    0  ...   \n",
       "3543  1.0   1.0   0       1.0   0   0    2     0        1.0    0  ...   \n",
       "3544  1.0   1.0   0       1.0   0   0    0     0        1.0    0  ...   \n",
       "3545  1.0   1.0   0       1.0   0   0    0     0        1.0    0  ...   \n",
       "\n",
       "      Ill  slummer  Rashomon  dipsticks  Bearable  Staggeringly    ’  \\\n",
       "0     1.0      1.0       1.0        1.0       1.0           1.0  1.0   \n",
       "1     1.0      1.0       1.0        1.0       1.0           1.0  1.0   \n",
       "2     1.0      1.0       1.0        1.0       1.0           1.0  1.0   \n",
       "3     1.0      1.0       1.0        1.0       1.0           1.0  1.0   \n",
       "4     1.0      1.0       1.0        1.0       1.0           1.0  1.0   \n",
       "...   ...      ...       ...        ...       ...           ...  ...   \n",
       "3541  1.0      1.0       1.0        1.0       1.0           1.0  1.0   \n",
       "3542  1.0      1.0       1.0        1.0       1.0           1.0  1.0   \n",
       "3543  1.0      1.0       1.0        1.0       1.0           1.0  1.0   \n",
       "3544  1.0      1.0       1.0        1.0       1.0           1.0  1.0   \n",
       "3545  1.0      1.0       1.0        1.0       1.0           1.0  1.0   \n",
       "\n",
       "       ve  muttering  dissing  \n",
       "0     1.0        1.0      1.0  \n",
       "1     1.0        1.0      1.0  \n",
       "2     1.0        1.0      1.0  \n",
       "3     1.0        1.0      1.0  \n",
       "4     1.0        1.0      1.0  \n",
       "...   ...        ...      ...  \n",
       "3541  1.0        1.0      1.0  \n",
       "3542  1.0        1.0      1.0  \n",
       "3543  1.0        1.0      1.0  \n",
       "3544  1.0        1.0      1.0  \n",
       "3545  1.0        1.0      1.0  \n",
       "\n",
       "[3546 rows x 20756 columns]"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "products['ispos'] = (products.sentiment > 0).astype(int)\n",
    "products['predicted_ispositive'] = 1\n",
    "df_product_bows.fillna(1,inplace=True)\n",
    "# for i in df_product_bows.columns:\n",
    "#     print(i)\n",
    "#     print(\"\")\n",
    "df_product_bows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4218838127467569\n"
     ]
    }
   ],
   "source": [
    "products['predicted_ispositive'] = nb.predict(df_product_bows.values).astype(int)\n",
    "\n",
    "products.head()\n",
    "print((products.predicted_ispositive == products.ispos).sum() / len(products))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>ispos</th>\n",
       "      <th>predicted_ispositive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1_1</td>\n",
       "      <td>-0.90</td>\n",
       "      <td>troubleshooting ad-2500 and ad-2600 no picture...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1_2</td>\n",
       "      <td>-0.15</td>\n",
       "      <td>repost from january 13, 2004 with a better fit...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1_3</td>\n",
       "      <td>-0.20</td>\n",
       "      <td>does your apex dvd player only play dvd audio ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1_4</td>\n",
       "      <td>-0.10</td>\n",
       "      <td>or does it play audio and video but scrolling ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1_5</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>before you try to return the player or waste h...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3541</td>\n",
       "      <td>309_4</td>\n",
       "      <td>-1.80</td>\n",
       "      <td>the other day when i was listening to a song, ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3542</td>\n",
       "      <td>309_5</td>\n",
       "      <td>-1.30</td>\n",
       "      <td>it says i have a harddisk problem.</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3543</td>\n",
       "      <td>309_6</td>\n",
       "      <td>-1.95</td>\n",
       "      <td>and since i'm out here i can't mail it back un...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3544</td>\n",
       "      <td>309_7</td>\n",
       "      <td>0.45</td>\n",
       "      <td>it worked good for a while.</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3545</td>\n",
       "      <td>309_8</td>\n",
       "      <td>-2.75</td>\n",
       "      <td>it did lock up on me a couple of times, and th...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3546 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id  sentiment  \\\n",
       "0       1_1      -0.90   \n",
       "1       1_2      -0.15   \n",
       "2       1_3      -0.20   \n",
       "3       1_4      -0.10   \n",
       "4       1_5      -0.50   \n",
       "...     ...        ...   \n",
       "3541  309_4      -1.80   \n",
       "3542  309_5      -1.30   \n",
       "3543  309_6      -1.95   \n",
       "3544  309_7       0.45   \n",
       "3545  309_8      -2.75   \n",
       "\n",
       "                                                   text  ispos  \\\n",
       "0     troubleshooting ad-2500 and ad-2600 no picture...      0   \n",
       "1     repost from january 13, 2004 with a better fit...      0   \n",
       "2     does your apex dvd player only play dvd audio ...      0   \n",
       "3     or does it play audio and video but scrolling ...      0   \n",
       "4     before you try to return the player or waste h...      0   \n",
       "...                                                 ...    ...   \n",
       "3541  the other day when i was listening to a song, ...      0   \n",
       "3542                 it says i have a harddisk problem.      0   \n",
       "3543  and since i'm out here i can't mail it back un...      0   \n",
       "3544                        it worked good for a while.      1   \n",
       "3545  it did lock up on me a couple of times, and th...      0   \n",
       "\n",
       "      predicted_ispositive  \n",
       "0                        0  \n",
       "1                        0  \n",
       "2                        0  \n",
       "3                        0  \n",
       "4                        0  \n",
       "...                    ...  \n",
       "3541                     0  \n",
       "3542                     0  \n",
       "3543                     0  \n",
       "3544                     0  \n",
       "3545                     0  \n",
       "\n",
       "[3546 rows x 5 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "products"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "2346796cae19bac5a2861f52d2390d006b13795fa7268dc74325b7da2fa2a240"
  },
  "kernelspec": {
   "display_name": "Python 3.8.5 64-bit",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
